/**
	@file   StringTokenize.h
	@date   2010/09/08
	@author sang-soo kim
	@brief  String tokenizer
*/

#if  !defined(__STRING_TOKENIZER_H__)
#define __STRING_TOKENIZER_H__

#include <vector>
#include <string>
#include <stdint.h>
#include "Character.h"

class StringTokenizer
{
public:
	static inline void tokenize_by_char(const std::string& line, std::vector< Character  >& VToken)
	{
        unsigned int i;
        const uint8_t *ch;

        Character token;
        for(i=0;i<line.length();)
        {
            token.clear();

            ch = (const uint8_t *)(line.c_str()+i);

            if( (  (*ch)&0xf0 ) == 0xf0 )
            {
                token.setValue( line.c_str()+i,4 ); 
                i=i+4;
            }
            else if( (  (*ch)&0xe0 ) == 0xe0 )
            {
                token.setValue( line.c_str()+i,3 );
                i=i+3;
            }
            else if( ( (*ch)&0x80 ) == 0x80 )
            {
                i++;
                continue;
            }
            else
            {
                token.setValue( line.c_str()+i,1 );
                i++;
            }
            VToken.push_back( token );
        }

	}
	static inline void tokenize(const std::string& str, std::vector<std::string>& tokens, const std::string& delimiters = " ")
	{
		// Skip delimiters at beginning.
		std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
		// Find first "non-delimiter".
		std::string::size_type pos     = str.find_first_of(delimiters, lastPos);

		while (std::string::npos != pos || std::string::npos != lastPos)
		{
			// Found a token, add it to the vector.
			tokens.push_back(str.substr(lastPos, pos - lastPos));
			// Skip delimiters.  Note the "not_of"
			lastPos = str.find_first_not_of(delimiters, pos);
			// Find next "non-delimiter"
			pos = str.find_first_of(delimiters, lastPos);
		}

	}

};

#endif

